#encoding:utf-8
import requests
from bs4 import BeautifulSoup
import random
import urllib
import urllib.request
import re
import csv

import datetime
import pymysql
import sys

# 数据库配置
host = '127.0.0.1' # localhost:本地数据库
user = 'root' # 用户名
password = '12345678' # 密码
database = 'weather_crawler' # 数据库名称
charset = 'utf8' # 编码方式

# 创建数据库连接
conn = pymysql.connect(host=host, user=user, password=password
                           , database=database, charset=charset)
cursor = conn.cursor(cursor=pymysql.cursors.DictCursor)

headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'
}
urllist=[]
def get_data(city_id):
	file = open('nanchang_weather.csv','w',newline='')
	csv_writer = csv.writer(file)
	csv_writer.writerow(["日期","星期","最高温度","最低温度","天气","风向","城市ID"])
	sql = "insert into crawler_citydata (publish_time,week,top_temperature,low_temperature,weather,wind_direction,city_id) values (%s,%s,%s,%s,%s,%s,%s)"
	for url in urllist:
		response = requests.get(url,headers=headers)
		soup = BeautifulSoup(response.text, 'html.parser')
		weather_list = soup.select('div[class="tian_three"]')
		for weather in weather_list:
			ul_list = weather.select('ul')
			for ul in ul_list:
				li_list= ul.select('li')
				str=""
				for li in li_list:
					div_list = li.select('div')
					list_div = []
					for div in div_list:
						row_data = div.string
						if "℃" in row_data:
							row_data = row_data.strip("℃")
							list_div.append(row_data)
						elif " 星期" in row_data:
							data = row_data.split(" ")[0]
							week = row_data.split(" ")[1]
							list_div.append(data)
							list_div.append(week)
						else:
							list_div.append(row_data)
					list_div.append(city_id)
					csv_writer.writerow(list_div)
					cursor.execute(sql,tuple(list_div))
					conn.commit()
	file.close()
def main(area,year,city_id):
	for year in range(year,year+1):
		for month in range(1,13):
			urllist.append("http://lishi.tianqi.com/%s/%d%02d.html"%(area,year,month))
	get_data(city_id)

if __name__ == '__main__':
	area = sys.argv[1]
	year = sys.argv[2]
	city_id = sys.argv[3]
	main(area,int(year),int(city_id))